# -*- coding:utf8 -*-

import re, sys, json, datetime, random, time
from scrapy.selector import Selector
from scrapy.exceptions import CloseSpider
import urllib
import lxml.html
import importlib
try:
    from scrapy.spiders import Spider
except:
    from scrapy.spider import BaseSpider as Spider

from scrapy.http import Request, FormRequest
from scrapy.utils.response import get_base_url
from scrapy.utils.url import urljoin_rfc

from gaokaopai.items import *
from gaokaopai.dao import *
from gaokaopai.util import *

importlib.reload(sys)
# sys.setdefaultencoding("utf-8")

class AdmissionSpider(Spider):
    name        = 'gaokaopai_admission'
    allow       = ['gaokaopai.com']

    def __init__(self, *args, **kwargs):
        super(AdmissionSpider, self).__init__(*args, **kwargs)

    def start_requests(self):
        yield Request("http://www.gaokaopai.com/fenshuxian-sct-3.html", callback=self.parse_list, dont_filter=True)

    def parse_list(self, response):
        for a_dom in response.xpath(u"//dt/strong[contains(text(), '生源地')]/parent::dt/following-sibling::dd/span/a[not(contains(text(), '不限'))]"):
            provincename = ''.join(a_dom.xpath(u"./text()").extract()).strip()

            url = ''.join(a_dom.xpath(u"./@href").extract()).strip()

            yield Request(url, callback=self.parse_year, meta={'provincename':provincename}, dont_filter=True)

    def parse_year(self, response):
        base_url = get_base_url(response)
        meta = response.meta

        for tr_dom in response.xpath(u"//div[contains(@class, 'markTableItem')][3]/table/.//tr"):
            year = ''.join(tr_dom.xpath("./td[1]/text()").extract()).strip()
            provincename = ''.join(tr_dom.xpath("./td[2]/text()").extract()).strip()
            type = ''.join(tr_dom.xpath("./td[3]/text()").extract()).strip()
            name = ''.join(tr_dom.xpath("./td[4]/text()").extract()).strip()
            score = ''.join(tr_dom.xpath("./td[5]/text()").extract()).strip()

            '''
            table       = Field() # t_admission
    provinceid  = Field()
    provincename= Field()
    year        = Field()
    type        = Field()
    name        = Field()
    score       = Field()
            '''

            province = getProvince(provincename)
            if province != None:
                ad                  = Admission()
                ad['table']         = 't_admission'
                ad['provinceid']    = province['id']
                ad['provincename']  = provincename
                ad['year']          = year
                ad['type']          = type
                ad['name']          = name
                ad['score']         = score

                yield ad


        '''分页效果'''
        next_page = ''.join(response.xpath(u"//div[contains(@class, 'pager')]/a[contains(text(), '下一页')]/@href").extract())
        if next_page != '':
            yield Request(urljoin_rfc(base_url, next_page), callback=self.parse_year, dont_filter=True)

